get_value_from_env(){
    # @arg1: name
    # @arg2: default value
    if [ "${!1}" == '' ]
    then
        export $1=$2
    fi
}

get_value_from_env PROC_TITLE testADSLab
get_value_from_env INTERVAL_SEC 5
echo PROC_TITLE=$PROC_TITLE
echo INTERVAL_SEC=$INTERVAL_SEC

get_net(){
    NET_STATUS=`ifstat 1 1`
}
get_cpumem(){
    CPUMEM_STATUS=`top -n 1 -b | grep -E "${PROC_TITLE}"`
}
get_gpu(){
    OUTPUT_NVIDIA_SMI=`nvidia-smi`
    GPUMessage=`echo -e "${OUTPUT_NVIDIA_SMI}" | grep % | awk '{print $9,$10,$11,$12,$13}'`
}
get_timestamp(){
    current=`date "+%Y-%m-%d %H:%M:%S"`  
    timestamp=`date -d "$current" +%s`   
}


echo "Wait until $PROC_TITLE appears in top"
get_cpumem
while [ "$CPUMEM_STATUS" == '' ]
do
    sleep 1
    get_cpumem
done

echo "Keep printing hardware status every $INTERVAL_SEC seconds"
while [ "$CPUMEM_STATUS" != '' ]
do
    echo --------- TOP -----------
    echo -e TOP=$CPUMEM_STATUS
    cputotal=`echo -e "$CPUMEM_STATUS"｜tr -s " "| awk '{sum+=$9} END {print sum}'`
    memtotal=`echo -e "$CPUMEM_STATUS"｜tr -s " "| awk '{sum+=$10} END {print sum}'`
    get_gpu
    get_net
    get_timestamp
    echo --------- CPU ------------
    echo total %CPU=$cputotal
    echo --------- MEM ------------
    echo total %MEM=$memtotal
    echo --------- NET ------------
    echo "$NET_STATUS"
    echo --------- GPU ------------
    echo "$GPUMessage"
    echo --------- NVIDIA_SMI -----
    echo "${OUTPUT_NVIDIA_SMI}"
    echo --------- TIME -----------
    echo timestamp=$timestamp
    echo time=$current
    echo 
    sleep $INTERVAL_SEC
    get_cpumem
done

echo "Scripts Finished"
exit
